#!/usr/bin/env python
# -*- coding: utf8 -*-
from multiprocessing import Pool
import sys, os
import unittest
import requests
import re

def worker(url, folder='new'):
    item = url
    headers={'Referer': url}
    rpos = item.rfind('/')
    down_url = item[:rpos].replace('file.php', 'down.php')
    print down_url
    tid = item[rpos+1:][:-5]
    data = {'type': 'torrent',
            'name': tid,
            'id': tid}
    response = requests.post(down_url, data=data, headers=headers)
    with open(os.path.join(folder, tid+'.torrent'), 'wb') as fp:
        for chunk in response.iter_content(chunk_size=1024):
            if chunk:
                fp.write(chunk)

def down(urls):
    needs = []
    sn = requests.session()
    for url in urls:
        sn.proxies = {'http': 'http://127.0.0.1:8087/'}
        content = sn.get(url).content
        links = re.findall(r'href="(http://[^"]+)', content)
        needs.extend([link for link in links if 'file.php' in link])
    return needs

if __name__ == '__main__':
    needs = down(sys.argv[1:])
    print needs
    pool = Pool(processes=5)
    pool.map(worker,  needs)
    pool.close()
    pool.join()
